#include<iostream>
#include<string>
#include<vector>
#include<boost/filesystem.hpp>
#include"util.hpp"
const std::string src_path="data/input/";
const std::string raw_path="data/raw_html/raw.txt";
static bool ParseTitle(std::string &file, std::string *out);
static bool ParseContent(std::string &file, std::string *out);
static bool ParseUrl(const std::string &path,std::string *out);
typedef struct DocInfol
{
	std::string title;
	std::string content;
	std::string url;//官网url
}DocInfo_t;
bool EnumFile(const std::string &src_path,std::vector<std::string> &files_list);
bool ParselFile(std::vector<std::string> &files_list,std::vector<DocInfo_t> &results);
bool SaveFile( std::vector<DocInfo_t> &results,const std::string &raw_path);
int main()
{
	

	std::vector<std::string> files_list;
	//读取并解析
	if(!EnumFile(src_path,files_list)){
		std::cerr<<"enum file name error!"<<std::endl;
		return 1;
	}
	//解析
	
	std::vector<DocInfo_t> results;
	if(!ParselFile(files_list,results)){
		std::cerr<<"parse file name error!"<<std::endl;
		return 2;
	}
	//存储
	if(!SaveFile(results,raw_path)){
		std::cerr<<"parse file name error!"<<std::endl;
		return 2;
	}
	std::cout <<"yes";
	return 0;
}

bool EnumFile(const std::string &src_path,std::vector<std::string> &files_list){
	namespace fs=boost::filesystem;
	fs::path root_pash(src_path);
	if(!fs::exists(root_pash)){
	std::cerr<<src_path<<"not path"<<std::endl;
	return false;
	}
	fs::recursive_directory_iterator end;//定义一个空的
	for(fs::recursive_directory_iterator i(root_pash);i!=end; i++){
		if(!fs::is_regular_file(*i)){
			continue;
		}
		if(i->path().extension()!=".html"){
			continue;
		}
		//路径合法
		//std::cout << i->path().string()<<std::endl;
		files_list.push_back(i->path().string());
	}

	return true;
};
static bool ParseTitle(std::string &file, std::string *out){
            std::size_t begin=file.find("<title>");
			if(begin==std::string::npos){
				return false;
			}
			begin+=7;
			std::size_t end=file.find("</title>");
			if(end==std::string::npos){
				return false;
			}
			if(begin<end)
			*out=file.substr(begin,end-begin);
			else
			return false;

			return true;

}

static bool ParseContent(std::string &file, std::string *out){
	enum status{//状态机
		in,
		need
	};
	enum status flag = in;
	for(char c:file){
		switch (flag)
		{
		case in:
			if(c=='>')flag=need;
			break;
		case need:
			if(c=='<')flag=in;
			else{
				if(c=='\n')c=' ';//不保留文件中的\n，\n分割文件
			out->push_back(c);
			}
			break;
		default:
			break;
		}
	}

	
	return true;
}
static bool ParseUrl(const std::string &path,std::string *out){
			std::string url_head="https://www.boost.org/doc/libs/1_78_0/soc/html/";
			std::string url_end=path.substr(src_path.size());
			*out=url_head+url_end;
			return true;
}
bool ParselFile(std::vector<std::string> &files_list,std::vector<DocInfo_t> &results){
	for(std::string &path:files_list){
		std::string result;
		if(!ns_util::FileUtill::ReadFile(path,&result)){
			continue;
		}
		//提取
		DocInfo_t doc;
		if(!ParseTitle(result,&doc.title)){
			continue;
		}
		if(!ParseContent(result,&doc.content)){
			continue;
		}
		if(!ParseUrl(path,&doc.url)){
			continue;
		}
		
		/*
		std::cout << doc.title<<std::endl;
		std::cout << doc.url<<std::endl;
		std::cout << doc.content<<std::endl;*/
		results.push_back(std::move(doc));
		
	}
	return true;
};
bool SaveFile( std::vector<DocInfo_t> &results,const std::string &raw_path){
	std::ofstream out(raw_path, std::ios::out|std::ios::binary);
	if(!out.is_open()){
		std::cerr << "open failed!"<<raw_path<<std::endl;
		return false;
	}
	for(auto &i:results){
		std::string out_str;
		out_str = i.title;
		out_str += '\3';
		out_str += i.url;
		out_str += '\3';
		out_str += i.content;
		out_str += '\n';
		//std::cout<<i.title<<std::endl;
		out.write(out_str.c_str(),out_str.size());
	}
	out.close();
	return true;
};
